{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import lightgbm as lgb\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import KFold\n",
    "import datetime\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas.api.types import is_datetime64_any_dtype as is_datetime\n",
    "from pandas.api.types import is_categorical_dtype\n",
    "\n",
    "def reduce_mem_usage(df, use_float16=False):\n",
    "    \"\"\"\n",
    "    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    \n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n",
    "            continue\n",
    "        col_type = df[col].dtype\n",
    "        \n",
    "        if col_type != object:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == \"int\":\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype(\"category\")\n",
    "\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n",
    "    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train= pd.read_csv('../../Large_output/train_clean_merge.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def features_engineering(df):\n",
    "    \n",
    "    # Sort by localtime\n",
    "    df.sort_values(\"local_time\")\n",
    "    df.reset_index(drop=True)\n",
    "    \n",
    "    # Add more features\n",
    "    df[\"local_time\"] = pd.to_datetime(df[\"local_time\"],format=\"%Y-%m-%d %H:%M:%S\")\n",
    "    df[\"hour\"] = df[\"local_time\"].dt.hour\n",
    "    df[\"weekend\"] = df[\"local_time\"].dt.weekday\n",
    "    df['square_feet'] =  np.log1p(df['square_feet'])\n",
    "    \n",
    "    \n",
    "    # Encode Categorical Data\n",
    "    le = LabelEncoder()\n",
    "    df[\"primary_use\"] = le.fit_transform(df[\"primary_use\"])\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 2638.86 MB\n",
      "Memory usage after optimization is: 733.78 MB\n",
      "Decreased by 72.2%\n"
     ]
    }
   ],
   "source": [
    "df_train = reduce_mem_usage(df_train,use_float16=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_engineer = features_engineering(df_train)\n",
    "train_engineer.loc[(train_engineer['site_id']==0) & (train_engineer['meter']==0),'meter_reading']\\\n",
    "=train_engineer.loc[(train_engineer['site_id']==0) & (train_engineer['meter']==0),'meter_reading'].mul(0.2931)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = np.log1p(df_train[\"meter_reading\"])\n",
    "features = df_train[['building_id', 'meter','site_id','primary_use', 'square_feet','air_temperature',\\\n",
    "                    'cloud_coverage','dew_temperature','precip_depth_1_hr','hour', 'weekend','is_holiday']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# xgb bayesian: \n",
    "import xgboost as xgb\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from bayes_opt import BayesianOptimization\n",
    "dtrain = xgb.DMatrix(features, label=target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run on gpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def xgb_evaluate(max_depth, subsample,gamma, colsample_bytree, max_leaves,max_bin, min_child_weight,colsample_bylevel,\\\n",
    "                reg_alpha,reg_lambda, random_seed=6):\n",
    "    params = {'eval_metric': 'rmse',\\\n",
    "              'objective': 'reg:linear',\\\n",
    "              'booster':'gbtree',\\\n",
    "              'max_depth': int(max_depth),\\\n",
    "              'subsample': subsample,\\\n",
    "              'eta': 0.05,\\\n",
    "              'tree_method':'gpu_hist',\\\n",
    "              'gamma': gamma,\\\n",
    "              'colsample_bytree': colsample_bytree,\\\n",
    "              'max_leaves': int(max_leaves),\\\n",
    "              'max_bin':int(max_bin),\\\n",
    "              'min_child_weight':min_child_weight,\\\n",
    "              'colsample_bylevel':colsample_bylevel,\\\n",
    "              'reg_alpha':reg_alpha,\\\n",
    "              'reg_lambda':reg_lambda,\n",
    "              'n_gpus': 2}\n",
    "    cv_result = xgb.cv(params, dtrain, num_boost_round=1000, nfold=3,seed=random_seed, stratified=False, verbose_eval=100,early_stopping_rounds=50)    \n",
    "    # Bayesian optimization only knows how to maximize, not minimize, so return the negative RMSE\n",
    "    return -1.0 * cv_result['test-rmse-mean'].iloc[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|   iter    |  target   | colsam... | colsam... |   gamma   |  max_bin  | max_depth | max_le... | min_ch... | reg_alpha | reg_la... | subsample |\n",
      "-------------------------------------------------------------------------------------------------------------------------------------------------\n",
      "[09:49:29] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[09:49:29] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[09:49:30] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[0]\ttrain-rmse:4.12314+0.0003706\ttest-rmse:4.12314+0.000676327\n",
      "[100]\ttrain-rmse:1.39528+0.00200797\ttest-rmse:1.39561+0.00249345\n",
      "[200]\ttrain-rmse:1.26164+0.00378422\ttest-rmse:1.26224+0.00386222\n",
      "[300]\ttrain-rmse:1.19906+0.00115681\ttest-rmse:1.19987+0.000617367\n",
      "[400]\ttrain-rmse:1.1564+0.00218372\ttest-rmse:1.1574+0.00200922\n",
      "[500]\ttrain-rmse:1.12336+0.00814927\ttest-rmse:1.12458+0.0075568\n",
      "[600]\ttrain-rmse:1.09848+0.00801517\ttest-rmse:1.09987+0.00742283\n",
      "[700]\ttrain-rmse:1.08005+0.00944957\ttest-rmse:1.08159+0.00898835\n",
      "[800]\ttrain-rmse:1.06308+0.00765239\ttest-rmse:1.0648+0.00711621\n",
      "[900]\ttrain-rmse:1.047+0.00934657\ttest-rmse:1.04888+0.00886026\n",
      "[999]\ttrain-rmse:1.03257+0.00904562\ttest-rmse:1.03462+0.0086239\n",
      "| \u001b[0m 1       \u001b[0m | \u001b[0m-1.035   \u001b[0m | \u001b[0m 0.1537  \u001b[0m | \u001b[0m 0.5253  \u001b[0m | \u001b[0m 0.6118  \u001b[0m | \u001b[0m 331.7   \u001b[0m | \u001b[0m 9.77    \u001b[0m | \u001b[0m 1.805e+0\u001b[0m | \u001b[0m 6.381   \u001b[0m | \u001b[0m 1.43    \u001b[0m | \u001b[0m 1.793   \u001b[0m | \u001b[0m 0.6589  \u001b[0m |\n",
      "[09:55:26] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[09:55:26] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[09:55:27] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[0]\ttrain-rmse:4.12382+0.000339117\ttest-rmse:4.12382+0.000706657\n",
      "[100]\ttrain-rmse:1.81062+0.000340592\ttest-rmse:1.81062+0.000800606\n",
      "[200]\ttrain-rmse:1.74878+0.000287876\ttest-rmse:1.74878+0.000758991\n",
      "[300]\ttrain-rmse:1.71623+0.000287238\ttest-rmse:1.71623+0.000723556\n",
      "[400]\ttrain-rmse:1.6925+0.000260543\ttest-rmse:1.69251+0.000702655\n",
      "[500]\ttrain-rmse:1.66841+0.000264866\ttest-rmse:1.66842+0.000662813\n",
      "[600]\ttrain-rmse:1.65691+0.000257663\ttest-rmse:1.65692+0.000646708\n",
      "[700]\ttrain-rmse:1.64367+0.000263853\ttest-rmse:1.64368+0.000619362\n",
      "[800]\ttrain-rmse:1.63189+0.000266599\ttest-rmse:1.6319+0.000607701\n",
      "[900]\ttrain-rmse:1.62105+0.000278319\ttest-rmse:1.62107+0.000579435\n",
      "[999]\ttrain-rmse:1.61725+0.000278103\ttest-rmse:1.61726+0.000577559\n",
      "| \u001b[0m 2       \u001b[0m | \u001b[0m-1.617   \u001b[0m | \u001b[0m 0.3097  \u001b[0m | \u001b[0m 0.3203  \u001b[0m | \u001b[0m 0.02416 \u001b[0m | \u001b[0m 400.9   \u001b[0m | \u001b[0m 1.319   \u001b[0m | \u001b[0m 1.74e+03\u001b[0m | \u001b[0m 18.44   \u001b[0m | \u001b[0m 0.3767  \u001b[0m | \u001b[0m 1.276   \u001b[0m | \u001b[0m 0.7594  \u001b[0m |\n",
      "[09:57:24] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[09:57:24] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[09:57:25] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[0]\ttrain-rmse:4.11781+0.000312893\ttest-rmse:4.11781+0.000731979\n",
      "[100]\ttrain-rmse:1.25693+0.00127961\ttest-rmse:1.25724+0.00131034\n",
      "[200]\ttrain-rmse:1.12604+0.0030709\ttest-rmse:1.12653+0.00250752\n",
      "[300]\ttrain-rmse:1.06046+0.000837734\ttest-rmse:1.06109+0.000570402\n",
      "[400]\ttrain-rmse:1.02134+0.00203046\ttest-rmse:1.02207+0.00112595\n",
      "[500]\ttrain-rmse:0.993843+0.00143573\ttest-rmse:0.99469+0.00054306\n",
      "[600]\ttrain-rmse:0.967051+0.000176205\ttest-rmse:0.968032+0.000987917\n",
      "[700]\ttrain-rmse:0.945869+0.000875084\ttest-rmse:0.946957+0.00109833\n",
      "[800]\ttrain-rmse:0.929088+0.00147424\ttest-rmse:0.93031+0.00072879\n",
      "[900]\ttrain-rmse:0.914446+0.00109863\ttest-rmse:0.915773+0.000300623\n",
      "[999]\ttrain-rmse:0.902326+0.00147844\ttest-rmse:0.903764+0.000745588\n",
      "| \u001b[95m 3       \u001b[0m | \u001b[95m-0.9038  \u001b[0m | \u001b[95m 0.5389  \u001b[0m | \u001b[95m 0.5294  \u001b[0m | \u001b[95m 0.7565  \u001b[0m | \u001b[95m 315.5   \u001b[0m | \u001b[95m 7.104   \u001b[0m | \u001b[95m 1.859e+0\u001b[0m | \u001b[95m 12.82   \u001b[0m | \u001b[95m 0.1149  \u001b[0m | \u001b[95m 1.517   \u001b[0m | \u001b[95m 0.7209  \u001b[0m |\n",
      "[10:02:49] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[10:02:50] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[10:02:51] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[0]\ttrain-rmse:4.12185+0.000303395\ttest-rmse:4.12185+0.000718904\n",
      "[100]\ttrain-rmse:1.27653+0.0101331\ttest-rmse:1.27753+0.00980367\n",
      "[200]\ttrain-rmse:1.15353+0.0037198\ttest-rmse:1.15531+0.00330028\n",
      "[300]\ttrain-rmse:1.09279+0.00765009\ttest-rmse:1.0952+0.00749895\n",
      "[400]\ttrain-rmse:1.05642+0.00968151\ttest-rmse:1.05939+0.00895942\n",
      "[500]\ttrain-rmse:1.02452+0.0138488\ttest-rmse:1.02798+0.0132121\n",
      "[600]\ttrain-rmse:0.997591+0.0121998\ttest-rmse:1.00156+0.0115398\n",
      "[700]\ttrain-rmse:0.974651+0.0107927\ttest-rmse:0.979075+0.0100539\n",
      "[800]\ttrain-rmse:0.955632+0.0115056\ttest-rmse:0.960465+0.0107784\n",
      "[900]\ttrain-rmse:0.939515+0.011326\ttest-rmse:0.944759+0.010574\n",
      "[999]\ttrain-rmse:0.923009+0.00952644\ttest-rmse:0.928641+0.00879086\n",
      "| \u001b[0m 4       \u001b[0m | \u001b[0m-0.9286  \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 180.0   \u001b[0m | \u001b[0m 12.0    \u001b[0m | \u001b[0m 1.835e+0\u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 0.1     \u001b[0m |\n",
      "[10:11:14] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[10:11:14] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[10:11:15] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[0]\ttrain-rmse:4.12187+0.000318513\ttest-rmse:4.12187+0.000695672\n",
      "[100]\ttrain-rmse:1.2816+0.00488857\ttest-rmse:1.28242+0.00544539\n",
      "[200]\ttrain-rmse:1.16544+0.000645773\ttest-rmse:1.16701+0.00138254\n",
      "[300]\ttrain-rmse:1.10344+0.00245499\ttest-rmse:1.10565+0.00328101\n",
      "[400]\ttrain-rmse:1.06914+0.00187565\ttest-rmse:1.07181+0.00274514\n",
      "[500]\ttrain-rmse:1.04144+0.00112346\ttest-rmse:1.04457+0.000755954\n",
      "[600]\ttrain-rmse:1.01756+0.00347788\ttest-rmse:1.02112+0.00274223\n",
      "[700]\ttrain-rmse:0.995586+0.00421185\ttest-rmse:0.999591+0.00349724\n",
      "[800]\ttrain-rmse:0.973731+0.00605896\ttest-rmse:0.978118+0.00531909\n",
      "[900]\ttrain-rmse:0.957538+0.00605927\ttest-rmse:0.962324+0.00530667\n",
      "[999]\ttrain-rmse:0.941832+0.00485219\ttest-rmse:0.946986+0.00407802\n",
      "| \u001b[0m 5       \u001b[0m | \u001b[0m-0.947   \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 180.0   \u001b[0m | \u001b[0m 12.0    \u001b[0m | \u001b[0m 1e+03   \u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 0.1     \u001b[0m |\n",
      "[10:19:48] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[10:19:49] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[10:19:50] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[0]\ttrain-rmse:4.12648+0.000331111\ttest-rmse:4.12648+0.000720242\n",
      "[100]\ttrain-rmse:1.59222+0.00175221\ttest-rmse:1.59232+0.00194581\n",
      "[200]\ttrain-rmse:1.50704+0.00186952\ttest-rmse:1.50718+0.00202219\n",
      "[300]\ttrain-rmse:1.46997+0.00260436\ttest-rmse:1.47015+0.0022865\n",
      "[400]\ttrain-rmse:1.45197+0.00296723\ttest-rmse:1.45218+0.00243769\n",
      "[500]\ttrain-rmse:1.43922+0.00306907\ttest-rmse:1.43945+0.00242224\n",
      "[600]\ttrain-rmse:1.4337+0.00307469\ttest-rmse:1.43394+0.00249488\n",
      "[700]\ttrain-rmse:1.42795+0.00283999\ttest-rmse:1.42821+0.00223412\n",
      "[800]\ttrain-rmse:1.4215+0.00240219\ttest-rmse:1.42177+0.00189775\n",
      "[900]\ttrain-rmse:1.41915+0.00237994\ttest-rmse:1.41943+0.00195347\n",
      "[999]\ttrain-rmse:1.41706+0.00227243\ttest-rmse:1.41734+0.00194674\n",
      "| \u001b[0m 6       \u001b[0m | \u001b[0m-1.417   \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 500.0   \u001b[0m | \u001b[0m 12.0    \u001b[0m | \u001b[0m 1e+03   \u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.9     \u001b[0m |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[10:26:46] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[10:26:47] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[10:26:48] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[0]\ttrain-rmse:4.12191+0.000350515\ttest-rmse:4.12191+0.00066408\n",
      "[100]\ttrain-rmse:1.27258+0.00496985\ttest-rmse:1.2735+0.00481093\n",
      "[200]\ttrain-rmse:1.15728+0.00521147\ttest-rmse:1.15893+0.0052561\n",
      "[300]\ttrain-rmse:1.09291+0.00361559\ttest-rmse:1.09518+0.00316121\n",
      "[400]\ttrain-rmse:1.05661+0.00574123\ttest-rmse:1.05941+0.00539948\n",
      "[500]\ttrain-rmse:1.02624+0.00727025\ttest-rmse:1.02947+0.00700512\n",
      "[600]\ttrain-rmse:1.00122+0.00704845\ttest-rmse:1.00492+0.00709764\n",
      "[700]\ttrain-rmse:0.977006+0.00545869\ttest-rmse:0.981163+0.0054098\n",
      "[800]\ttrain-rmse:0.957564+0.00632136\ttest-rmse:0.962168+0.00645766\n",
      "[900]\ttrain-rmse:0.939693+0.0052722\ttest-rmse:0.944736+0.00542688\n",
      "[999]\ttrain-rmse:0.925171+0.00535174\ttest-rmse:0.930638+0.00544016\n",
      "| \u001b[0m 7       \u001b[0m | \u001b[0m-0.9306  \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 180.0   \u001b[0m | \u001b[0m 12.0    \u001b[0m | \u001b[0m 1.281e+0\u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 0.1     \u001b[0m |\n",
      "[10:35:17] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[10:35:17] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[10:35:18] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[0]\ttrain-rmse:4.12649+0.000316229\ttest-rmse:4.12649+0.000760686\n",
      "[100]\ttrain-rmse:1.63709+0.000215974\ttest-rmse:1.63712+0.00103579\n",
      "[200]\ttrain-rmse:1.56042+0.000346307\ttest-rmse:1.56047+0.00101263\n",
      "[300]\ttrain-rmse:1.52615+0.000349441\ttest-rmse:1.52621+0.000822845\n",
      "[400]\ttrain-rmse:1.5101+0.000178692\ttest-rmse:1.51016+0.000984385\n",
      "[500]\ttrain-rmse:1.50055+0.000272129\ttest-rmse:1.50061+0.000941928\n",
      "[600]\ttrain-rmse:1.49628+0.000382508\ttest-rmse:1.49635+0.000886181\n",
      "[700]\ttrain-rmse:1.49244+0.000438844\ttest-rmse:1.49251+0.000956179\n",
      "[800]\ttrain-rmse:1.48875+0.000420191\ttest-rmse:1.48883+0.000881228\n",
      "[900]\ttrain-rmse:1.48716+0.000448177\ttest-rmse:1.48724+0.000862061\n",
      "[999]\ttrain-rmse:1.48592+0.000431309\ttest-rmse:1.486+0.000897219\n",
      "| \u001b[0m 8       \u001b[0m | \u001b[0m-1.486   \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 244.8   \u001b[0m | \u001b[0m 12.0    \u001b[0m | \u001b[0m 2e+03   \u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 2.0     \u001b[0m | \u001b[0m 0.1     \u001b[0m |\n",
      "[10:39:36] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[10:39:37] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[10:39:38] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[0]\ttrain-rmse:4.09862+0.00033924\ttest-rmse:4.09864+0.000688916\n",
      "[100]\ttrain-rmse:0.852663+0.00178225\ttest-rmse:0.855429+0.000955538\n",
      "[200]\ttrain-rmse:0.770111+0.000448256\ttest-rmse:0.775102+0.000649517\n",
      "[300]\ttrain-rmse:0.732839+0.00132988\ttest-rmse:0.739767+0.00059033\n",
      "[400]\ttrain-rmse:0.70758+0.00115822\ttest-rmse:0.716349+0.000439503\n",
      "[500]\ttrain-rmse:0.690438+0.000882122\ttest-rmse:0.701113+0.000554938\n",
      "[600]\ttrain-rmse:0.677315+0.000911044\ttest-rmse:0.689859+0.000338688\n",
      "[700]\ttrain-rmse:0.666151+0.00100448\ttest-rmse:0.680546+0.000464347\n",
      "[800]\ttrain-rmse:0.656376+0.000460181\ttest-rmse:0.672699+0.000618456\n",
      "[900]\ttrain-rmse:0.647628+0.000517617\ttest-rmse:0.665907+0.000787952\n",
      "[999]\ttrain-rmse:0.640159+0.00052629\ttest-rmse:0.66033+0.000575297\n",
      "| \u001b[95m 9       \u001b[0m | \u001b[95m-0.6603  \u001b[0m | \u001b[95m 0.9     \u001b[0m | \u001b[95m 0.9     \u001b[0m | \u001b[95m 1.0     \u001b[0m | \u001b[95m 180.0   \u001b[0m | \u001b[95m 12.0    \u001b[0m | \u001b[95m 1.537e+0\u001b[0m | \u001b[95m 3.0     \u001b[0m | \u001b[95m 0.1     \u001b[0m | \u001b[95m 0.1     \u001b[0m | \u001b[95m 0.9     \u001b[0m |\n",
      "[10:57:55] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[10:57:56] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[10:57:57] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[0]\ttrain-rmse:4.09844+0.000368928\ttest-rmse:4.09845+0.000653557\n",
      "[100]\ttrain-rmse:0.838916+0.00159476\ttest-rmse:0.841781+0.000750849\n",
      "[200]\ttrain-rmse:0.76112+0.00185833\ttest-rmse:0.766162+0.00129746\n",
      "[300]\ttrain-rmse:0.723029+0.000397957\ttest-rmse:0.730007+0.000383269\n",
      "[400]\ttrain-rmse:0.699667+0.000662563\ttest-rmse:0.708516+0.000303338\n",
      "[500]\ttrain-rmse:0.681454+0.00107533\ttest-rmse:0.692257+0.000379786\n",
      "[600]\ttrain-rmse:0.668466+0.000332028\ttest-rmse:0.681089+0.000356358\n",
      "[700]\ttrain-rmse:0.657788+0.000508968\ttest-rmse:0.672192+0.000432579\n",
      "[800]\ttrain-rmse:0.648569+0.000693197\ttest-rmse:0.664818+0.000294104\n",
      "[900]\ttrain-rmse:0.640224+0.000703751\ttest-rmse:0.65837+0.000240817\n",
      "[999]\ttrain-rmse:0.632949+0.0007013\ttest-rmse:0.652945+0.000233337\n",
      "| \u001b[95m 10      \u001b[0m | \u001b[95m-0.6529  \u001b[0m | \u001b[95m 0.9     \u001b[0m | \u001b[95m 0.9     \u001b[0m | \u001b[95m 1.0     \u001b[0m | \u001b[95m 180.0   \u001b[0m | \u001b[95m 0.0     \u001b[0m | \u001b[95m 1.655e+0\u001b[0m | \u001b[95m 20.0    \u001b[0m | \u001b[95m 0.1     \u001b[0m | \u001b[95m 0.1     \u001b[0m | \u001b[95m 0.9     \u001b[0m |\n",
      "[11:17:54] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:17:55] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:17:57] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[0]\ttrain-rmse:4.09714+0.000365422\ttest-rmse:4.09716+0.000666287\n",
      "[100]\ttrain-rmse:0.824335+0.000880136\ttest-rmse:0.82808+0.00088004\n",
      "[200]\ttrain-rmse:0.748613+0.000958167\ttest-rmse:0.7551+0.00163216\n",
      "[300]\ttrain-rmse:0.714117+0.00153208\ttest-rmse:0.722978+0.00213749\n",
      "[400]\ttrain-rmse:0.689643+0.00183794\ttest-rmse:0.700841+0.00228382\n",
      "[500]\ttrain-rmse:0.672533+0.000998751\ttest-rmse:0.685977+0.00134856\n",
      "[600]\ttrain-rmse:0.658563+0.00127859\ttest-rmse:0.674325+0.00108476\n",
      "[700]\ttrain-rmse:0.646993+0.000832965\ttest-rmse:0.665056+0.000733525\n",
      "[800]\ttrain-rmse:0.636983+0.000971649\ttest-rmse:0.657352+0.0007036\n",
      "[900]\ttrain-rmse:0.627851+0.000827329\ttest-rmse:0.650563+0.00089928\n",
      "[999]\ttrain-rmse:0.619875+0.000536422\ttest-rmse:0.644908+0.00070136\n",
      "| \u001b[95m 11      \u001b[0m | \u001b[95m-0.6449  \u001b[0m | \u001b[95m 0.9     \u001b[0m | \u001b[95m 0.9     \u001b[0m | \u001b[95m 1.0     \u001b[0m | \u001b[95m 500.0   \u001b[0m | \u001b[95m 12.0    \u001b[0m | \u001b[95m 2e+03   \u001b[0m | \u001b[95m 3.0     \u001b[0m | \u001b[95m 0.1     \u001b[0m | \u001b[95m 0.1     \u001b[0m | \u001b[95m 0.9     \u001b[0m |\n",
      "[11:39:17] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:39:18] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:39:19] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[0]\ttrain-rmse:4.09976+0.000520521\ttest-rmse:4.09977+0.00109226\n",
      "[100]\ttrain-rmse:0.866613+0.00153361\ttest-rmse:0.868954+0.00134657\n",
      "[200]\ttrain-rmse:0.77958+0.00171366\ttest-rmse:0.783799+0.000967406\n",
      "[300]\ttrain-rmse:0.737161+0.00178895\ttest-rmse:0.743235+0.00108815\n",
      "[400]\ttrain-rmse:0.71119+0.00100255\ttest-rmse:0.719311+0.000439346\n",
      "[500]\ttrain-rmse:0.69386+0.000455757\ttest-rmse:0.703999+0.000525621\n",
      "[600]\ttrain-rmse:0.681394+0.000343826\ttest-rmse:0.693594+0.00043284\n",
      "[700]\ttrain-rmse:0.671008+0.00031681\ttest-rmse:0.68532+0.000437846\n",
      "[800]\ttrain-rmse:0.662596+0.00044568\ttest-rmse:0.679025+0.000435679\n",
      "[900]\ttrain-rmse:0.65535+0.000298248\ttest-rmse:0.673833+0.000540215\n",
      "[999]\ttrain-rmse:0.648811+0.000180408\ttest-rmse:0.669319+0.000562113\n",
      "| \u001b[0m 12      \u001b[0m | \u001b[0m-0.6693  \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 1.0     \u001b[0m | \u001b[0m 500.0   \u001b[0m | \u001b[0m 12.0    \u001b[0m | \u001b[0m 1.299e+0\u001b[0m | \u001b[0m 3.0     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.1     \u001b[0m |\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[11:55:32] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:55:33] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[11:55:34] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "[0]\ttrain-rmse:4.09881+0.000269857\ttest-rmse:4.09882+0.000740935\n",
      "[100]\ttrain-rmse:0.86566+0.0011386\ttest-rmse:0.868008+0.00160907\n",
      "[200]\ttrain-rmse:0.78165+0.000440528\ttest-rmse:0.785763+0.00117285\n",
      "[300]\ttrain-rmse:0.740664+0.0017635\ttest-rmse:0.746295+0.00260847\n",
      "[400]\ttrain-rmse:0.71583+0.0011753\ttest-rmse:0.722903+0.00173396\n",
      "[500]\ttrain-rmse:0.69637+0.00183903\ttest-rmse:0.704926+0.0020122\n",
      "[600]\ttrain-rmse:0.68202+0.00143813\ttest-rmse:0.692061+0.00153493\n",
      "[700]\ttrain-rmse:0.670476+0.00125857\ttest-rmse:0.682002+0.00123233\n",
      "[800]\ttrain-rmse:0.660016+0.00117765\ttest-rmse:0.673081+0.00120619\n",
      "[900]\ttrain-rmse:0.651486+0.00100987\ttest-rmse:0.666026+0.000918536\n",
      "[999]\ttrain-rmse:0.643945+0.00103792\ttest-rmse:0.659999+0.00097396\n",
      "| \u001b[0m 13      \u001b[0m | \u001b[0m-0.66    \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 0.9     \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 371.5   \u001b[0m | \u001b[0m 0.0     \u001b[0m | \u001b[0m 1.315e+0\u001b[0m | \u001b[0m 20.0    \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.1     \u001b[0m | \u001b[0m 0.9     \u001b[0m |\n",
      "=================================================================================================================================================\n"
     ]
    }
   ],
   "source": [
    "xgb_bo = BayesianOptimization(xgb_evaluate, {'max_depth': (0, 12), \\\n",
    "                                             'gamma': (0, 1),\\\n",
    "                                             'subsample': (0.1, 0.9),\\\n",
    "                                            'max_leaves': (1000, 2000),\\\n",
    "                                            'colsample_bytree': (0.1, 0.9),\\\n",
    "                                            'reg_lambda': (0.1, 2),\\\n",
    "                                            'reg_alpha': (0.1, 2),\\\n",
    "                                            'max_bin':(180,500),\\\n",
    "                                            'colsample_bylevel':(0.1,0.9),\\\n",
    "                                            'min_child_weight': (3, 20)})\n",
    "# Use the expected improvement acquisition function to handle negative numbers\n",
    "xgb_bo.maximize(init_points=3, n_iter=10, acq='ei')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "|   iter    |  target   | colsam... | colsam... |   gamma   |  max_bin  | max_depth | max_le... | min_ch... | reg_alpha | reg_la... | subsample |\n",
    "|  11       | -0.6449   |  0.9      |  0.9      |  1.0      |  500.0    |  12.0     |  2e+03    |  3.0      |  0.1      |  0.1      |  0.9      |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'eval_metric': 'rmse',\\\n",
    "              'objective': 'reg:squarederror',\\\n",
    "              'booster':'gbtree',\\\n",
    "              'nthread' : 4,\\\n",
    "              'eta' : 0.05,\\\n",
    "              'max_leaves': 2000,\\\n",
    "              'max_depth' : 12,\\\n",
    "              'subsample' : 0.9,\\\n",
    "              'colsample_bytree' : 0.9,\\\n",
    "              'colsample_bylevel' : 0.9,\\\n",
    "             'gamma':1.0,\\\n",
    "             'max_bin':500,\\\n",
    "             'min_child_weight':3.0,\\\n",
    "             'reg_alpha':0.1,\\\n",
    "             'reg_lambda':0.1}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
